# -*- coding: utf-8 -*-
import sys 
reload(sys) 
sys.setdefaultencoding("utf-8") 
import re
import scrapy
from scrapy.selector import Selector
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from renren.utils import *
from scrapy.contrib.linkextractors import LinkExtractor
from scrapy.http import Request, FormRequest
from renren.items import Profile
from scrapy.contrib.loader import ItemLoader

def fst(m):
    return m.group(1)

def log_page(response, filename):
        with open(filename, 'w') as f:
            f.write("%s\n%s\n%s\n" % (response.url, response.headers, response.body))

class renrenSpider(scrapy.Spider) :
    name = "renren2"
    allowed_domains = ["www.renren.com"]


    start_urls = [
        


"http://www.renren.com/291995759/profile",

    ]

    def start_requests(self):
        return [Request("http://www.renren.com/SysHome.do",callback = self.post_login)]

   
    def post_login(self, response):
    	log_page(response, "renren_login.html")
        return [FormRequest.from_response(response,   
        	 				
                            formdata = {
                            'email': 'pkunetworklab1@163.com',
                            'password': 'pku123456'
                            },
                            callback = self.after_login,
                            )]
    def after_login(self,response):
    	for url in self.start_urls:            
            yield self.make_requests_from_url(url)
    
    def parse(self, response):
        def get_field(response, s):
            return response.xpath(s).extract()
        l = Profile()
#        print "haha "+response.url
        print response.url
        if not is_matching_main_page(response.url):
            return

    	log_page(response,'renren_after.html')

        try:
            l['school'] = response.xpath('//li[@class="school"]/span/text()').extract()[0][3:]
        except:
            l['school'] =''
        if not is_pku(l['school']):
            return
        l['name'] = response.xpath('//h1[@class="avatar_title"]/text()').extract()[0]
        l['numberOfFriends'] = response.xpath('//div[@class="has-friend"]/h4/span/text()').extract()[0]
        l['numberOfViews'] = re.findall('^[^0-9]*([0-9]*).*$', response.xpath('//h1[@class="avatar_title"]/span/text()').extract()[0], re.S)[0]
        try:
            l['gender'] = response.xpath('//li[@class="birthday"]/span[1]/text()').extract()[0]
        except:
            l['gender'] = ''
        l['url'] = response.url
        l['ID']= response.url.split('/')[-2]
    	l['image_urls'] = response.xpath('//img[@id="userpic"]/@src').extract()
        l['department'] = get_field(response, '//div[@id="educationInfo"]/div[@class="info-section-info"]/dl[1]/dd[1]/a[last()]/text()')
        l['birthyear'] = get_field(response, '//div[@id="basicInfo"]/div[@class="info-section-info"]/dl[2]/dd[1]/a[1]/text()')
        l['birthmonth'] = get_field(response, '//div[@id="basicInfo"]/div[@class="info-section-info"]/dl[2]/dd[1]/a[2]/text()')
        l['birthday'] = get_field(response, '//div[@id="basicInfo"]/div[@class="info-section-info"]/dl[2]/dd[1]/a[3]/text()')
        l['constellation'] = get_field(response, '//div[@id="basicInfo"]/div[@class="info-section-info"]/dl[2]/dd[1]/a[4]/text()')
        l['hometown'] = get_field(response, '//div[@id="basicInfo"]/div[@class="info-section-info"]/dl[3]/dd[1]/text()')
        yield l